package u1;

import static java.lang.System.*;
import static u1.Util.*;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

import org.xml.sax.SAXException;

/**
 * Takes two arguments: (1) output file stem, (2) (regularized) titles list
 * (uncompressed) (see Titles.java). Builds an adjacency list from uncompressed
 * wiki data and saves it in a set of files based on the first argument to this
 * program. The "ordinal" of an article is its index in the (regularized) titles
 * list, starting at zero. If the first argument is "adj" then the generated
 * files are:
 * 
 * <pre>
 * adj      : the forward links for all pages (see Util.writeMultiMap).
 * adj.dead : list of (regularized) titles of forward links for which no page
 *            was found; this might be because there is no page, or because we 
 *            weren't smart enough to find it. One title per line.
 * adj.redirect: list of ordinals that were marked as redirect pages (no
 *            content of their own) and the ordinal of the page redirected to).
 *            One ordinal per line.
 * </pre>
 */
public class ForwardLinks {
  public static final String EXT_DEAD = ".dead";
  public static final String EXT_FORWARD = "";
  public static final String EXT_REDIRECT = ".redirect";

  public static void main(String[] argv) throws IOException, SAXException {
    if (argv.length != 2) {
      err.println("Consult comments for usage.");
      exit(-1);
    }
    String adjFile = argv[0];
    String titlesFile = argv[1];

    /*
     * Read titles first. The assigned ordinals correspond to the order of the
     * titles in the titles file.
     */
    trace("Reading titles from %s...", titlesFile);
    BufferedReader titles = new BufferedReader(new InputStreamReader(
        new FileInputStream(new File(titlesFile)), "UTF-8"));
    final Map<String, Integer> ordinals = readStringIndex(titles);
    titles.close();
    trace("Read %d titles.", ordinals.size());

    /*
     * Now read in text to extract links. There's no need to keep all this in
     * memory, though it seems to be possible using the 64 bit Java on the
     * Eureka cluster nodes (8GB RAM) (but we spend quite a lot of time in GC).
     */

    // Write dead links to file as we go.
    final PrintStream adj = new PrintStream(new File(adjFile
        + ForwardLinks.EXT_FORWARD));
    final PrintStream redirect = new PrintStream(new File(adjFile
        + EXT_REDIRECT));
    final PrintStream dead = new PrintStream(new File(adjFile + EXT_DEAD),
        "UTF-8");

    trace("Reading pages from stdin...");
    final AdjWriter writer = new AdjWriter(adj);
    final Set<String> forwardLinks = new HashSet<String>();
    final HashSet<Integer> forwardLinkOrds = new HashSet<Integer>();
    PageReader reader = new PageReader() {

      @Override
      protected void readPage(String title, String text) {
        Integer titleOrd = ordinals.get(title);
        // We should find all the titles, but don't crash.
        if (titleOrd == null) {
          trace("Couldn't find title %s (page index %d)", title, getPageIndex());
          return;
        }
        
        // Strip comments, non-markup sections out of text.
        text = Util.stripCommentsAndNonMarkupSections(text);

        // If the page is a redirect, remember it separately.
        String redirectLink = Util.findRedirect(text);
        if (redirectLink != null) {
          Integer redirectOrd = ordinals.get(canonicalizeLink(redirectLink));
          if (redirectOrd == null) {
            // Not much we can do about a dead redirect.
            dead.println(redirectLink);
            return;
          } else {
            redirect.println(String.format("%d %d", titleOrd, redirectOrd));
          }
        }

        // Find internal links (which will include the #REDIRECT link).
        forwardLinks.clear();
        Util.findInternalLinks(text, forwardLinks);

        // Find the ordinals of the forward links; a link may not point to any
        // page, in which case we leave it out (but we keep track of it for
        // debugging (and interest) purposes).
        forwardLinkOrds.clear();
        for (String link : forwardLinks) {
          Integer linkOrd = ordinals.get(canonicalizeLink(link));
          if (linkOrd == null) {
            dead.println(link);
          } else {
            forwardLinkOrds.add(linkOrd);
          }
        }
        writer.write(titleOrd, forwardLinkOrds);

        // This can take a while; give feedback.
        if ((getPageIndex() + 1) % 100000 == 0)
          trace("Found forward links for %d pages.", getPageIndex() + 1);
      }
    };
    reader.parse(in);
    writer.close();
    redirect.close();
    dead.close();

    trace("Read %d pages. Done.", reader.getPageIndex());
  }
}
/*
* Copyright (c) 2009 John Lees-Miller
* 
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
* 
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* 
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/

